package com.effvi.jsoup.spider;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 网络蜘蛛
 * jsoup实现的图片批量抓取
 * @author kelvin.zhang
 *
 */
public class Spider {
	
    public static void main( String[] args ) throws IOException, URISyntaxException
    {
    	Document doc = Jsoup.connect("http://www.163.com").get();
    	Elements hrefs = doc.select("a");
    	Iterator<Element> it_hrefs = hrefs.iterator();
    	int out = 0;
    	int in = 0;
    	while(it_hrefs.hasNext()){
    		out++;
    		in = 0;
    		Element singleEle = it_hrefs.next();
    		Pattern p = Pattern.compile("https?://[^$|/|\\?]+");
    		Matcher m = p.matcher(singleEle.attr("href"));
    		String RootPath = "";
    		if(m.find()){
    			RootPath = m.group(0);
    		}
    		
    		String h = singleEle.attr("href");
    		System.out.println(h);
    		Document singleDoc = null;
			try {
				singleDoc = Jsoup.connect(h).get();
			} catch (Exception e1) {
				singleDoc = null;
			}
			if(singleDoc==null){
				continue;
			}
    		Elements imgs = singleDoc.select("img");
    		if(imgs!=null && imgs.size() > 0){
    			Iterator<Element> it = imgs.iterator();
    	    	while(it.hasNext()){
    	    		in++;
    	    		Element e = it.next();
    	    		String url = e.attr("src");
    	    		if(url !=null && !"".equals(url)){
    	    			if(!url.startsWith("http")){
    	    				if(!url.startsWith("/")){
    	    					url = RootPath +"/"+ url;
    	    				}else{
    	    					url = RootPath + url;
    	    				}
    	    				
    	    			}
    	    			System.out.println(url);
    	    			URL urlObj = new URL(url);
    	    			URLConnection connection = urlObj.openConnection();
    	    			InputStream is =  connection.getInputStream();
    	    			byte[] bs = new byte[1024];
    	    			int len;
    	    			String name = "img-163-"+out+"-"+in+".jpg";
    	    			OutputStream os = new FileOutputStream(name);
    	    			while((len = is.read(bs)) != -1){
    	    				os.write(bs, 0, len);
    	    			}
    	    			if(is!=null){
    	    				is.close();
    	    			}
    	    			if(os!=null){
    	    				os.close();
    	    			}
    	    			System.out.println(name);
    	    		}
    	    	}
    		}
    	}
    }
}
